package com.qen.yanshen

import cn.hutool.core.io.file.FileWriter
import cn.hutool.core.lang.Console
import cn.hutool.core.util.StrUtil
import cn.hutool.poi.excel.ExcelUtil
import it.skrape.core.htmlDocument
import it.skrape.fetcher.HttpFetcher
import it.skrape.fetcher.response
import it.skrape.fetcher.skrape
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.Semaphore
import java.util.concurrent.atomic.AtomicInteger

const val YanShenUrl = "https://onehu.xyz/archives"
const val BaseYanShenUrl = "https://onehu.xyz"
const val MaxPage = 301
const val UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
const val ZhihuOutFile = "/Users/a1/Documents/Git/Kotlin/Eleven/out/zhihu/"
const val THREAD_COUNT = 5

class Article(
    var date: String,
    var title: String,
    var content: String,
    var url: String,
    val html: String,
)

fun main3() {
    /**爬虫线程队列*/
    val threads = ArrayList<Thread>()

    /**已爬取的新闻数（所谓“已爬取”，实际是从拿到链接就算）*/
    val newsFetchedCounter = AtomicInteger()

    /**爬取结束信号*/
    val newsTargetReachedSemaphore = Semaphore(THREAD_COUNT, THREAD_COUNT)

    /**待爬取队列非空信号*/
    val linkQueueAvailableSemaphore = Semaphore(Int.MAX_VALUE, Int.MAX_VALUE)
    /**放出一个信号*/
    linkQueueAvailableSemaphore.release()

    val list: MutableList<Article> = ArrayList()
    var urlPre = YanShenUrl
    var pageNum = AtomicInteger(1)
    for (i in 0..THREAD_COUNT) {
        val thread = Thread {
            while (true) {
                runBlocking {
                    linkQueueAvailableSemaphore.acquire()
                }
                if (pageNum.get() >= MaxPage) {
                    newsTargetReachedSemaphore.release()
                    break
                }

                val urlNew = if (pageNum.get() == 1) YanShenUrl else YanShenUrl + "/page/${pageNum.get()}/#board"
                val referer = if (pageNum.get() == 1) YanShenUrl else YanShenUrl + "/page/${pageNum.get()}/"
                Console.log(urlNew)
                try {
                    skrape(HttpFetcher) {
                        request {
                            url = urlNew
                            headers = initHeader(urlPre)
                            cookies = initCookie()
                            userAgent = UserAgent
                        }
                        response {
                            htmlDocument {
                                // 类选择器，选择class="quote"
                                ".list-group-item-action" {
                                    findAll {
                                        forEach {
                                            val url = it.attribute("href")
                                            val item = it.children
                                            val time = item.get(0).text
                                            val title = item.get(1).text
                                            val res = detail(goUrl = url, refererUrl = urlNew, title = title)
                                            list.add(
                                                Article(
                                                    url = BaseYanShenUrl + url,
                                                    date = time,
                                                    title = title,
                                                    content = res,
                                                    html = ""
                                                )
                                            )
                                        }
                                    }
                                }
                            }
                        }
                    }
                }catch (e: Exception){

                }
                pageNum.getAndIncrement()
                urlPre = referer
            }
        }
        thread.start()
        threads.add(thread)
    }
    runBlocking {
        newsTargetReachedSemaphore.acquire()
    }
    threads.forEach { t -> t.join() }
    val writer = ExcelUtil.getWriter(ZhihuOutFile + "/Zhihu.xlsx")
    writer.passCurrentRow()
//    writer.merge(quoteList.size - 1, "标题")
    writer.write(list, true)
    writer.close()

}

fun main1() {
    detail(
        "/2024/10/18/%E6%9C%89%E4%B8%80%E4%B8%AA%E5%A4%96%E7%A7%91%E5%8C%BB%E7%94%9F%E7%94%B7%E6%9C%8B%E5%8F%8B%E6%98%AF%E7%A7%8D%E6%80%8E%E6%A0%B7%E7%9A%84%E4%BD%93%E9%AA%8C%EF%BC%9F.md",
        "https://onehu.xyz/archives/", "title"
    )
}

fun main2() {
    var list = listOf("a", "b", "c")
    Console.log(list.joinToString("\n"))
    val writer = FileWriter("/Users/a1/Documents/Git/Kotlin/Eleven/out/aaa.txt")
    writer.write(list.joinToString("\n"));
}

fun detail(goUrl: String, refererUrl: String, title: String): String {
    val urls = BaseYanShenUrl + goUrl
    var html = StringBuilder()
    var content = StringBuilder()
    var contentList = mutableListOf<String>()
    skrape(HttpFetcher) {
        request {
            url = urls
            headers = initHeader(refererUrl)
            cookies = initCookie()
            userAgent = UserAgent
        }
        response {
            htmlDocument {
                ".markdown-body" {
                    findAll {
                        forEach { t ->
                            html.append(t.html)
                            t.children.forEach { ii ->
                                content.append("${ii.text}\n")
                                contentList.add(ii.text)
                            }
                        }
                    }
                }
            }
        }
    }
//    Console.log("content:$content")
//    Console.log("Html:$html")
//    Console.log(contentList.joinToString("\n"))
    val writer = FileWriter(ZhihuOutFile + title + ".txt")
    writer.write(StrUtil.join("\n", contentList))
//    return mutableMapOf("content" to "./$title.txt", "html" to html.toString())
    return "./$title.txt"
}

fun initHeader(urlPre: String): MutableMap<String, String> {
    return mutableMapOf(
        "authority" to "onehu.xyz",
        "method" to "GET",
        "path" to "/archives/",
        "scheme" to "https",
        "accept" to "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "accept-encoding" to "deflate,br,zstd",
        "accept-language" to "zh-CN,zh;q=0.9,en;q=0.8",
        "cache-control" to "no-cache",
        "pragma" to "no-cache",
        "priority" to "u=0, i",
        "referer" to urlPre,
        "sec-ch-ua" to "\"Google Chrome\";v=\"129\", \"Not=A?Brand\";v=\"8\", \"Chromium\";v=\"129\"",
        "sec-ch-ua-mobile" to "?0",
        "sec-ch-ua-platform" to "macOS",
        "sec-fetch-dest" to "document",
        "sec-fetch-mode" to "navigate",
        "sec-fetch-site" to "same-origin",
        "sec-fetch-user" to "?1",
        "upgrade-insecure-requests" to "1",
    )
}

fun initCookie(): MutableMap<String, String> {
    return mutableMapOf(
        "_gid" to "GA1.2.1931293739.1729233785",
        "Hm_lvt_2d445f964e80c83e2275543c2a1d3bdd" to "1729233785",
        "HMACCOUNT" to "8E7727E55235B30C",
        "_gcl_au" to "1.1.1638383923.1729233785",
        "Hm_lpvt_2d445f964e80c83e2275543c2a1d3bdd" to "1729237216",
        "_ga_22RSYSQFPZ" to "GS1.1.1729237212.2.1.1729237215.0.0.0",
        "_ga" to "GA1.1.1988112821.1729233785",
        "__gads" to "ID=40910a19ea1dfdb5:T=1729233786:RT=1729237216:S=ALNI_MbHHhSUUEUVcB1j2gZTot3Q9wUHZg",
        "__gpi" to "UID=00000f48b69806cf:T=1729233786:RT=1729237216:S=ALNI_MaICBWBpttctbPmSm7MqgXUIvC47w",
        "__eoi" to "ID=6de62327d5d23592:T=1729233786:RT=1729237216:S=AA-AfjbLUYG8x4dH8JdKu8nyzrfU",
    )
}


fun main() {
    val list: MutableList<Article> = ArrayList()
    var urlPre = YanShenUrl
    for (i in 1..MaxPage) {
        val urlNew = if (i == 1) YanShenUrl else YanShenUrl + "/page/$i/#board"
        val referer = if (i == 1) YanShenUrl else YanShenUrl + "/page/$i/"
        try {
            skrape(HttpFetcher) {
                request {
                    url = urlNew
                    headers = initHeader(urlPre)
                    cookies = initCookie()
                    userAgent = UserAgent

//            proxy = proxyBuilder {
//                type = Proxy.Type.HTTP
//                host = "brd.superproxy.io"
//                port = 7890
//            }
//            authentication = basic {
//                username = "brd-customer-hl_4hgu8dwd-zone-residential"
//                password = "ZZZZZZZZZ"
//            }
                }
                response {
                    try {
                        htmlDocument {
                            // 类选择器，选择class="quote"
                            ".list-group-item-action" {
                                findAll {
                                    forEach {
                                        val url = it.attribute("href")
                                        val item = it.children
                                        val time = item.get(0).text
                                        val title = item.get(1).text
                                        val res = detail(goUrl = url, refererUrl = urlNew, title = title)
                                        list.add(
                                            Article(
                                                url = BaseYanShenUrl + url,
                                                date = time,
                                                title = title,
                                                content = res,
                                                html = ""
                                            )
                                        )
//                                Console.log("url:$url;\n time:$time; \n title:$title")
                                    }
                                }
                            }
                        }
                    }catch (e: Exception){
                        e.printStackTrace()
                    }
                }
            }
        }catch (e: Exception){
            e.printStackTrace()
        }
        urlPre = referer
    }
    val writer = ExcelUtil.getWriter(ZhihuOutFile + "/Zhihu.xlsx")
    writer.passCurrentRow()
//    writer.merge(quoteList.size - 1, "标题")
    writer.write(list, true)
    writer.close()

}